package com.algo.url;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 *  * multiple threads get title, key words, description from url list
 *   * @author lujianfeng@miaozhen.com
 *    */

public class TitleCrawler implements Runnable
{
	private static BufferedReader br;
	private static BufferedWriter bw;
	private static ScratchLogger logger;
	public static class EmptyTitleException extends IOException{
		private static final long serialVersionUID = 1L;
	}
	
	public static void close(){
		try{
			if(br != null) br.close();
			if(bw != null)	bw.close();
			if(logger != null)	logger.closeLogger();
		}catch(IOException e){
			throw new RuntimeException(e);
		}
	}
	
	public static String getTitle(String url){
		Document doc;
		
		try {
			new URL(url);
			Connection con = Jsoup.connect(url);
			con.userAgent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)");
			doc = con.get();
		}catch (IOException e) {
			if(logger != null)	logger.logExeption(url, e);
			return null;
		}
		
    	Elements heads = doc.getElementsByTag("head");
    	
    	StringBuilder sb = new StringBuilder();
    	
    	for(Element head : heads){
    		Elements titles = head.getElementsByTag("title");
    		for(Element title : titles){
    			sb.append(title.text().replaceAll("[\r\n\t]", " "));
    		}
    		
    		sb.append("\t");
    		Elements keys = head.getElementsByAttributeValue("name", "keywords");
    		for(Element key : keys){
    			sb.append(key.attr("content").replaceAll("[\r\n\t]", " "));
    		}
    		
    		sb.append("\t");
    		Elements descs = head.getElementsByAttributeValue("name", "description");
    		for(Element desc : descs){
    			sb.append(desc.attr("content").replaceAll("[\r\n\t]", " "));
    		}
    	}
    	
        String title =  sb.length() > 3 ? sb.toString() : null;
        if(logger != null){
	        if(title == null)	logger.logExeption(url, new EmptyTitleException());
	        else logger.afterScratchUrl(title);
        }
        return title;
	}
	
	public TitleCrawler(String in, String out, String log){
		try {
			if(br == null || bw == null)
				br = new BufferedReader(new FileReader(in));
			if(bw == null)
				bw = new BufferedWriter(new FileWriter(out));
			if(logger == null)
				logger = new ScratchLogger(log);
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	}
	
	
	
	public TitleCrawler(){}
	
	public void run(){
		while(true){
			try{
				String url = null;
				synchronized(br){
					url = br.readLine();
				}
				if(url == null)	break;
		    	String title = getTitle(url);
	    		if(title != null)
	    			synchronized(bw){
	    				bw.write(url + "\t" + title + "\n");
	    			}
			}catch(IOException e){
				throw new RuntimeException(e);
			}
		}
	}
	
	public void multiThreadsGetTitle(int threadsNum, int timeOutDays) throws InterruptedException{
	    	ExecutorService executor = Executors.newCachedThreadPool();
	    	for(int i = 0; i < threadsNum; i++)
	    		executor.execute(new TitleCrawler());
	    	executor.shutdown();
	    	System.out.println("INFO: " + threadsNum + " threads started at system time: "+ System.currentTimeMillis());
	    	executor.awaitTermination(timeOutDays, TimeUnit.DAYS);
	    	TitleCrawler.close();
	    	System.out.println("INFO: " + threadsNum + " threads closed at system time: "+ System.currentTimeMillis());
	}
	
	public static void main( String[] args ) throws InterruptedException{
    
    	
    		new TitleCrawler(args[0], args[1], args[2]).multiThreadsGetTitle(Integer.parseInt(args[3]), Integer.parseInt(args[4]));
    	    	
    		//new TitleCrawler("D://java//url50", "D://java//title", "D://java//log").multiThreadsGetTitle(1, 2);
    		logger.printScratchInfo();    	    	    	    
	}    	    	    	    	
}
